import os
import json
import importlib

import glob as gb
import utils as ut
import numpy as np
import pandas as pd
import seaborn as sns
import ipywidgets as pyw
import matplotlib.pyplot as plt

from IPython.display import clear_output

Global

%matplotlib inline

importlib.reload(ut)

plt.rcParams['text.usetex'] = True
plt.style.use(['dark_background'])

Data

df = pd.DataFrame()

for path_ground in sorted(gb.glob(os.path.join('results', 'v7', '**', 'ground-*.npy'), recursive=True)):
    path_directory = os.path.dirname(path_ground)
    path_parameters = gb.glob(os.path.join(path_directory, 'parameters-*.json'))[0]
    path_alpha = gb.glob(os.path.join(path_directory, 'alpha-*.npy'))[0]
    
    simulation = path_directory.split(os.sep)[-1]
    parameters = json.load(open(path_parameters))
    
    # ignore small views
    if parameters['view'] < 30:
        continue
    
    # ignore small forest's
    if parameters['size'] < 600:
        continue
    
    # filesize
    filesize_ground = os.path.getsize(path_ground) / 1024**2
    filesize_alpha = os.path.getsize(path_alpha) / 1024**2
    print(f'simulation: {simulation}, ground: {filesize_ground:.2f} MB, alpha: {filesize_alpha:.2f} MB')
    
    # load data
    alpha = np.load(path_alpha)
    ground = np.load(path_ground)
    
    # aggregate data
    scanned = np.count_nonzero(ground[:, :, 0])
    captured = np.count_nonzero(ground[:, :, 1])
    density = 1 - captured / scanned
    mean = 1 - (ground[:, :, 1] > 0).mean()
    trees = parameters['size']
    
    # append simulation data
    df = df.append(pd.json_normalize({ 
        'simulation': simulation,
        'scanned': scanned,
        'captured': captured,
        'density': density,
        'mean': mean / trees,
        **parameters,
        'data_ground': ground,
        'data_alpha': alpha
    }), ignore_index=True)

df = df.sort_values(['view', 'preset'], ascending=True)

simulation: AOS-Simulation-2021-06-08-12-05, ground: 1.00 MB, alpha: 91.00 MB
simulation: AOS-Simulation-2021-06-08-12-12, ground: 1.42 MB, alpha: 114.98 MB
simulation: AOS-Simulation-2021-06-08-12-25, ground: 2.04 MB, alpha: 144.73 MB
simulation: AOS-Simulation-2021-06-08-12-33, ground: 2.99 MB, alpha: 182.67 MB
simulation: AOS-Simulation-2021-06-08-12-40, ground: 4.59 MB, alpha: 234.12 MB
simulation: AOS-Simulation-2021-06-08-12-49, ground: 7.54 MB, alpha: 309.18 MB
simulation: AOS-Simulation-2021-06-08-12-56, ground: 13.92 MB, alpha: 431.41 MB
simulation: AOS-Simulation-2021-06-08-13-07, ground: 1.00 MB, alpha: 91.00 MB
simulation: AOS-Simulation-2021-06-08-13-17, ground: 1.42 MB, alpha: 114.98 MB
simulation: AOS-Simulation-2021-06-08-13-26, ground: 2.04 MB, alpha: 144.73 MB
simulation: AOS-Simulation-2021-06-08-13-36, ground: 2.99 MB, alpha: 182.67 MB
simulation: AOS-Simulation-2021-06-08-13-44, ground: 4.59 MB, alpha: 234.12 MB
simulation: AOS-Simulation-2021-06-08-13-56, ground: 7.54 MB, alpha: 309.18 MB
simulation: AOS-Simulation-2021-06-08-15-31, ground: 13.92 MB, alpha: 431.41 MB
simulation: AOS-Simulation-2021-06-08-15-49, ground: 1.00 MB, alpha: 91.00 MB
simulation: AOS-Simulation-2021-06-08-16-04, ground: 1.42 MB, alpha: 114.98 MB
simulation: AOS-Simulation-2021-06-08-16-18, ground: 2.04 MB, alpha: 144.73 MB
simulation: AOS-Simulation-2021-06-08-16-30, ground: 2.99 MB, alpha: 182.67 MB
simulation: AOS-Simulation-2021-06-08-16-43, ground: 4.59 MB, alpha: 234.12 MB
simulation: AOS-Simulation-2021-06-08-17-10, ground: 7.54 MB, alpha: 309.18 MB
simulation: AOS-Simulation-2021-06-08-17-26, ground: 13.92 MB, alpha: 431.41 MB

print('dataframe size:', df.shape)
df.filter(regex=r'^(?:(?!data).)*$').head(10)

dataframe size: (21, 19)

	simulation	scanned	captured	density	mean	speed	height	coverage	view	images	sampling	resolution	type	preset	size	ground	color
6	AOS-Simulation-2021-06-08-12-56	3644281	1407334	0.613824	0.001024	4	35	18.756443	30	356	1	512	monochrome	forest-12	600	70	16777215
13	AOS-Simulation-2021-06-08-15-31	3644281	365041	0.899832	0.001000	4	35	18.756443	30	356	1	512	monochrome	forest-13	900	70	16777215
20	AOS-Simulation-2021-06-08-17-26	3644281	40408	0.988912	0.000824	4	35	18.756443	30	356	1	512	monochrome	forest-14	1200	70	16777215
5	AOS-Simulation-2021-06-08-12-49	1967000	931785	0.526291	0.000881	4	35	25.477916	40	291	1	512	monochrome	forest-12	600	70	16777215
12	AOS-Simulation-2021-06-08-13-56	1967000	257870	0.868902	0.000966	4	35	25.477916	40	291	1	512	monochrome	forest-13	900	70	16777215
19	AOS-Simulation-2021-06-08-17-10	1967000	50044	0.974558	0.000812	4	35	25.477916	40	291	1	512	monochrome	forest-14	1200	70	16777215
4	AOS-Simulation-2021-06-08-12-40	1201216	736150	0.387163	0.000647	4	35	32.641536	50	309	1	512	monochrome	forest-12	600	70	16777215
11	AOS-Simulation-2021-06-08-13-44	1201216	223065	0.814301	0.000905	4	35	32.641536	50	309	1	512	monochrome	forest-13	900	70	16777215
18	AOS-Simulation-2021-06-08-16-43	1201216	40067	0.966645	0.000806	4	35	32.641536	50	309	1	512	monochrome	forest-14	1200	70	16777215
3	AOS-Simulation-2021-06-08-12-33	783225	561349	0.283285	0.000475	4	35	40.414519	60	222	1	512	monochrome	forest-12	600	70	16777215

Plots

Functions

def plot_alphas(df, mask, groups, title):
    
    # group dataframe
    df_group = df[['view', 'preset', 'data_alpha']][mask].groupby(groups[0])
    
    # subplots layout
    n_cols = 2
    n_rows = np.ceil(len(df_group) / n_cols).astype(np.int16)
    
    # initialize subplots
    fig, axs = plt.subplots(n_rows, n_cols, figsize=(20, n_rows * 7))
    
    # aggregate data
    for i, (name_0, group_0) in enumerate(df_group):
        
        ax = axs[i // 2, i % 2]
        for j, alphas in enumerate(group_0['data_alpha']):
            
            # print progress
            print(f'plot {title % name_0}: ', end='')
            
            # alpha dataframe
            df_alpha = ut.aggregate_alphas(alphas, sample=None)
            df_alpha['view'] = group_0['view'].iloc[j]
            df_alpha['preset'] = group_0['preset'].iloc[j]
            
            # filter corner alphas
            # df_alpha = df_alpha[df_alpha['alpha'] <= df_alpha['view'] / 2]
            
            # plot alphas
            for name_1, group_1 in df_alpha.groupby(groups[1]):
                print(f'{name_1}')
                group_1.plot(kind='line', x='alpha', y='ratio', label=f'{name_1} (visible/scanned)', ax=ax)
        
        # plot title and legend
        ax.set_title(title % name_0)
        ax.legend(title=groups[1])
    
    # clear print output
    clear_output()

def plot_grounds(df, mask, groups, title, subtitle):

    # group dataframe
    df_group = df[['view', 'preset', 'data_ground']][mask].groupby(groups[0])
    
    # aggregate data
    for i, (name_0, group_0) in enumerate(df_group):
        
        # subplots layout
        n_cols = 3
        n_rows = len(group_0)
        
        # initialize subplots
        fig, axs = plt.subplots(n_rows, n_cols, figsize=(27, n_rows * 7))
        
        for j, ground in enumerate(group_0['data_ground']):
            
            # print progress
            name_1 = subtitle % group_0[groups[1]].iloc[j]
            print(f'plot {title % name_0}: {name_1}')
            
            # plot ground images
            ax = axs[j] if n_rows > 1 else axs
            ut.plot_heatmap(ax[0], ground[:, :, 0], f'{name_1} - scanned pixels (count)')
            ut.plot_heatmap(ax[1], ground[:, :, 1], f'{name_1} - visible pixels (count)')
            ut.plot_heatmap(ax[2], ut.normalize_image(ground[:, :, 1] > 0), f'{name_1} - visible pixels (binary)')
        
        # plot title
        fig.suptitle(title % name_0)
    
    # clear print output
    clear_output()

filter_mask = (df['view'] >= 30) & (df['view'] <= 90) & (df['size'] >= 600)

Density

fig, axs = plt.subplots(1, 2, figsize=(20, 6))

columns = ['density', 'mean', 'captured', 'view', 'preset', 'size']
for i, (preset, group) in enumerate(df[columns][filter_mask].groupby('preset')):
    group.plot(kind='line', x='view', y='density', label=preset + ' (density)', ax=axs[0])
    group.plot(kind='line', x='view', y='mean', label=preset + ' (mean)', ax=axs[1])
    # group.plot(kind='barh', x='view', y='captured', label=preset + ' (captured)', color=f'C{i}', ax=axs[1])

Grounds

plot_grounds(df, filter_mask, ['preset', 'view'], 'preset %s', 'field of view %s°')

plot_grounds(df, filter_mask, ['view', 'preset'], 'field of view %s°', 'preset %s')

Alphas

plot_alphas(df, filter_mask, ['preset', 'view'], 'preset %s')

plot_alphas(df, filter_mask, ['view', 'preset'], 'field of view %s°')